# !/bin/bash -e

function start_mon() {
   echo "[$hostname] start ceph-mon in $hostname ..."
   sudo ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
   sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
   sudo mkdir -p /var/lib/ceph/bootstrap-osd
   sudo ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
   sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
   sudo ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
   sudo chown $USER:$USER /tmp/ceph.mon.keyring
   sudo chown -R $USER:$USER /var/lib/ceph
   sudo chown -R $USER:$USER /etc/ceph/
   sudo monmaptool --create --add $hostname $MON_HOST --fsid $FSID /tmp/monmap --clobber
   sudo -u $USER mkdir -p /var/lib/ceph/mon/ceph-$hostname
   sudo -u $USER ceph-mon --mkfs -i $hostname --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
   sudo ceph-mon -i $hostname
   echo "[$hostname] finish start ceph-mon in $hostname"

   # deal warning
   sudo ceph config set mon auth_allow_insecure_global_id_reclaim false
   sudo ceph mon enable-msgr2
}

function start_mgr() {
   echo "[$hostname] start ceph-mgr in $hostname ..."
   sudo mkdir -p /var/lib/ceph/mgr/ceph-$hostname
   sudo ceph auth get-or-create mgr.$hostname mon 'allow profile mgr' osd 'allow *' mds 'allow *' | sudo tee -a /var/lib/ceph/mgr/ceph-$hostname/keyring 
   sudo ceph-mgr -i $hostname
   echo "[$hostname] finish start ceph-mgr in $hostname"
}

function start_osd0() {
   echo "[$hostname] start ceph-osd in $hostname ..."
   OSDID=`uuidgen`
   OSD_SECRET=$(ceph-authtool --gen-print-key)
   OSD_NUM=$(echo "{\"cephx_secret\": \"$OSD_SECRET\"}" | \
      sudo ceph osd new $OSDID -i - \
      -n client.bootstrap-osd -k /var/lib/ceph/bootstrap-osd/ceph.keyring)
   echo "[$hostname] get osd.num = $OSD_NUM ..."
   sudo mkdir -p /var/lib/ceph/osd/ceph-$OSD_NUM
   sudo mkfs.xfs /dev/ceph-osds/osd$OSD_NUM -f > /dev/null
   sudo mount /dev/ceph-osds/osd$OSD_NUM /var/lib/ceph/osd/ceph-$OSD_NUM > /dev/null
   sudo ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-$OSD_NUM/keyring --name osd.$OSD_NUM --add-key $OSD_SECRET | sed "s/^/[$hostname] &/g"
   sudo ceph-osd -i $OSD_NUM --mkfs --osd-uuid $OSDID > /dev/null 2>&1
   sudo ceph-osd -i $OSD_NUM > /dev/null 2>&1
   echo "[$hostname] finish start ceph-osd.$OSD_NUM in $hostname"
}

function create_pools_for_rgw() {
   # ceph config set mon mon_max_pg_per_osd 1000
   ceph config set mgr mon_max_pg_per_osd 1000
   ceph osd pool create .rgw.root 32 32
   ceph osd pool create default.rgw.control 32 32
   ceph osd pool create default.rgw.log 32 32
   ceph osd pool create default.rgw.meta 32 32
   ceph osd pool create default.rgw.buckets.data 32 32
   ceph osd pool create default.rgw.buckets.index 32 32
   ceph osd pool application enable .rgw.root rgw
   ceph osd pool application enable default.rgw.control rgw
   ceph osd pool application enable default.rgw.log rgw
   ceph osd pool application enable default.rgw.meta rgw
   ceph osd pool application enable default.rgw.buckets.data rgw
   ceph osd pool application enable default.rgw.buckets.index rgw
}

function cp_conf_to_other_nodes() {
   # copy conf/keyring
   for node in $CLIENT $OSD2 $OSD3
   do
      scp /etc/ceph/ceph.client.admin.keyring root@${node}:/etc/ceph/
      scp /etc/ceph/ceph.conf root@${node}:/etc/ceph/
      ssh ${node} "sudo chown -R $USER:$USER /etc/ceph"
   done

   for node in $OSD2 $OSD3
   do 
      ssh ${node} "
         sudo mkdir -p /var/lib/ceph/bootstrap-osd/
         sudo chown -R $USER:$USER /var/lib/ceph
      "
      scp /var/lib/ceph/bootstrap-osd/ceph.keyring ${node}:/var/lib/ceph/bootstrap-osd/ceph.keyring
   done
}

function start_rgw() {
   echo "[client] start ceph-rgw in client ..."
   ssh client "
      sudo mkdir -p /var/log/radosgw
      sudo chown $USER:$USER /var/log/radosgw
      echo '[client.rgw.client]
      host=client
      keyring=/etc/ceph/keyring.radosgw.gateway
      log file=/var/log/radosgw/client.radosgw.gateway.log
      rgw_frontends = civetweb port=8080
      ' | sudo tee -a /etc/ceph/ceph.conf
      sudo ceph-authtool -C -n client.rgw.client --gen-key /etc/ceph/keyring.radosgw.gateway
      sudo chown $USER:$USER /etc/ceph/keyring.radosgw.gateway
      sudo ceph-authtool -n client.rgw.client --cap mon 'allow rw' --cap osd 'allow rwx' /etc/ceph/keyring.radosgw.gateway
      sudo ceph -k /etc/ceph/ceph.client.admin.keyring auth add client.rgw.client -i /etc/ceph/keyring.radosgw.gateway
      sudo radosgw -c /etc/ceph/ceph.conf -n client.rgw.client --keyring=/etc/ceph/keyring.radosgw.gateway
   "
   echo "[client] finish start ceph-rgw in client ..."
}

source IP.conf

echo "start deploy a single mon's cluster ..."
if [ ! -d "/etc/ceph" ]; then
	sudo mkdir /etc/ceph
fi

FSID=`uuidgen`
hostname=$(hostname -s)

echo "[global]
   fsid = $FSID
   cluster id = 1
   fabric kafka brokers = '10.10.9.40:9092'
   fabric kafka topic = 'ceph'
   fabric sdk addr = '10.10.9.40:8080'
   mon initial members = $hostname
   mon host = $MON_HOST
   public network = $NETWORK
   osd journal size = 1024
   osd pool default size = 3
   osd pool default min size = 2
   osd pool default pg num = 128
   osd pool default pgp num = 128
   osd crush chooseleaf type = 1
   mon allow pool delete = true
   # mon max pg per osd = 1000
" | sudo tee -a /etc/ceph/ceph.conf

start_mon
start_mgr
start_osd0
cp_conf_to_other_nodes
create_pools_for_rgw

# start osd in node2/3
for node in $OSD2 $OSD3
do
   ssh $node "\
   cd ~/ceph_fabric_config
   ./ceph_osd_deploy.sh"
done

start_rgw

# create s3 API user & save secret key
# radosgw-admin user list
# radosgw-admin user rm --uid="testuser"
radosgw-admin user create --uid="testuser" --display-name="First User" > ~/ceph_fabric_config/secret.json

echo "finish deploy a single mon's cluster!"